python: split source packages away from compiled packages

Well, they're not yet compiled, but in the next commit
they should be.

People have been complaining [citation needed] to me
via email or via Github that Python's performance is crap
because it packages sources directly and they're not compiled.
And Python has to compile the sources on each run, and
on-the-fly.

Allowing compilation caching is also a no-no, because
I'll get complaints that the flash storage fills up
whenever a Python app runs.

So, to give the user a choice, the new de-facto packaging
for Python packages will be:
* ship compiled + [ preferably ] optimized files
* package sources separately

The problem is that this doubles the number of packages
in LEDE/OpenWrt, but build-times should not suffer a big
hit, since the compilation is done once, and the
install phase should not be too intensive.

Oh, and people don't need ship source packages if
they don't want to.
To do that, a packager needs to just call
`$(eval $(call BuildPackage,python-<package>-src))`

The `python-` prefix is important.

Signed-off-by: Alexandru Ardelean <[email protected]>

diff --git a/lang/python/Makefile b/lang/python/Makefile
index 25d07fbd437fb9533c48c9b06e853c29ee68770a..a42a1870027d1a4be49c79d17059872b7821cd23 100644 (file)
--- a/lang/python/Makefile
+++ b/lang/python/Makefile
@@ -301,6 +301,7 @@ $(eval $(call HostBuild))
 $(foreach package, $(PYTHON_PACKAGES),  \
        $(eval $(call PyPackage,$(package))) \
        $(eval $(call BuildPackage,$(package))) \
+       $(eval $(call BuildPackage,$(package)-src)) \
 )
 
 $(eval $(call PyPackage,python-base))
@@ -312,3 +313,7 @@ $(eval $(call BuildPackage,python-pip-conf))
 $(eval $(call BuildPackage,python-base))
 $(eval $(call BuildPackage,python-light))
 $(eval $(call BuildPackage,python))
+
+$(eval $(call BuildPackage,python-base-src))
+$(eval $(call BuildPackage,python-light-src))
+$(eval $(call BuildPackage,python-src))

diff --git a/lang/python/files/python-package-install.sh b/lang/python/files/python-package-install.sh
index c3de0cc950ce912d7dbd15a431cffcd9562e7a56..a08f8b4153e6e0f97502149adc33a77fc01a0227 100644 (file)
--- a/lang/python/files/python-package-install.sh
+++ b/lang/python/files/python-package-install.sh
@@ -34,5 +34,37 @@ process_filespec() {
        )
 }
 
-process_filespec "$1" "$2" "$3"
+src_dir="$1"
+dst_dir="$2"
+python="$3"
+mode="$4"
+filespec="$5"
+
+process_filespec "$src_dir" "$dst_dir" "$filespec" || {
+       echo "process filespec error-ed"
+       exit 1
+}
+
+if [ "$mode" == "sources" ] ; then
+       # Copy only python source files
+       find $dst_dir -not -name "*\.py" | xargs rm -f
+       # Delete empty folders (if the case)
+       find $dst_dir/usr -type d | xargs rmdir &> /dev/null
+       rmdir $dst_dir/usr &> /dev/null
+       exit 0
+fi
+
+# XXX [So that you won't goof as I did]
+# Note: Yes, I tried to use the -O & -OO flags here.
+#       However the generated byte-codes were not portable.
+#       So, we just stuck to un-optimized byte-codes,
+#       which is still way better/faster than running
+#       Python sources all the time.
+$python -m compileall -d '/' $dst_dir || {
+       echo "python -m compileall err-ed"
+       exit 1
+}
+# Delete source files and pyc [ un-optimized bytecode files ]
+# We may want to make this optimization thing configurable later, but not sure atm
+find $dst_dir -name "*\.py" | xargs rm -f
 

diff --git a/lang/python/files/python-package.mk b/lang/python/files/python-package.mk
index 4f05dfebbc71a7878fc6c2cef5df745c0ce303f2..e9ec85c6f2ca84c545b11168f23e8c75ee8b1109 100644 (file)
--- a/lang/python/files/python-package.mk
+++ b/lang/python/files/python-package.mk
@@ -34,6 +34,17 @@ endif
 
 define PyPackage
 
+  define Package/$(1)-src
+    $(call Package/$(1))
+    TITLE+= (sources)
+    DEPENDS:=$$$$(foreach dep,$$$$(filter +python-%,$$$$(DEPENDS)),$$$$(dep)-src)
+  endef
+
+  define Package/$(1)-src/description
+    $(call Package/$(1)/description).
+    (Contains the Python sources for this package).
+  endef
+
   # Add default PyPackage filespec none defined
   ifndef PyPackage/$(1)/filespec
     define PyPackage/$(1)/filespec
@@ -58,16 +69,22 @@ define PyPackage
        if [ -e files/python-package-install.sh ] ; then \
                $(SHELL) files/python-package-install.sh \
                        "$(PKG_INSTALL_DIR)" "$$(1)" \
+                       "$(HOST_PYTHON_BIN)" "$$(2)" \
                        "$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" ; \
        elif [ -e $(STAGING_DIR)/mk/python-package-install.sh ] ; then \
                $(SHELL) $(STAGING_DIR)/mk/python-package-install.sh \
                        "$(PKG_INSTALL_DIR)" "$$(1)" \
+                       "$(HOST_PYTHON_BIN)" "$$(2)" \
                        "$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" ; \
        else \
                echo "No 'python-package-install.sh' script found" ; \
                exit 1 ; \
        fi
   endef
+
+  define Package/$(1)-src/install
+       $$(call Package/$(1)/install,$$(1),sources)
+  endef
 endef
 
 $(call include_mk, python-host.mk)